{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "def train_model(X_data, y_data, model):\n",
    "    X_model = X_data.to_numpy()\n",
    "    y_model = y_data.to_numpy().ravel()\n",
    "\n",
    "    imputer = SimpleImputer(strategy='median')\n",
    "    X_model = imputer.fit_transform(X_model)\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_model, y_model, test_size=0.2, random_state=42)\n",
    "    \n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    y_pred = model.predict(X_test)\n",
    "\n",
    "    f1 = f1_score(y_test, y_pred, average='micro')\n",
    "\n",
    "    print(\"F1 Score:\", f1)\n",
    "\n",
    "    return f1\n",
    "\n",
    "def predict_model(X_data, model):\n",
    "    X_model = X_data.to_numpy()\n",
    "\n",
    "    imputer = SimpleImputer(strategy='median')\n",
    "    X_model = imputer.fit_transform(X_model)\n",
    "\n",
    "    y_pred = model.predict(X_model)\n",
    "\n",
    "    return y_pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "X = pd.read_csv('features/X_train_features.csv').to_numpy()\n",
    "y = pd.read_csv('public/y_train.csv')['target'].to_numpy()\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Perform train-test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "\n",
    "# Create and train the Random Forest model with selected features\n",
    "model_selected = GradientBoostingClassifier(learning_rate=0.05, n_estimators=1000, max_depth=20, \n",
    "                                 min_samples_split=60, min_samples_leaf=20, subsample=1.0,\n",
    "                                 max_features=100, random_state=42)\n",
    "model_selected.fit(X_train, y_train.ravel())\n",
    "\n",
    "# Evaluate the model with selected features\n",
    "f1_selected = f1_score(y_test, model_selected.predict(X_test), average='micro')\n",
    "print(\"F1 Score with selected features:\", f1_selected)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import nevergrad as ng\n",
    "import numpy as np\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Define the objective function for Nevergrad optimization\n",
    "\n",
    "def objective_function(hyperparameters):\n",
    "    \n",
    "    # Create and train the SVC model\n",
    "    model = GradientBoostingClassifier(\n",
    "        n_estimators=hyperparameters[\"n_estimators\"],\n",
    "        learning_rate=hyperparameters[\"learning_rate\"],\n",
    "        subsample=hyperparameters[\"subsample\"],\n",
    "        max_depth=hyperparameters[\"max_depth\"],\n",
    "        min_samples_split=hyperparameters[\"min_samples_split\"],\n",
    "        min_samples_leaf=hyperparameters[\"min_samples_leaf\"],\n",
    "        max_features=hyperparameters[\"max_features\"],\n",
    "        random_state=42\n",
    "        )\n",
    "    model.fit(X_train, y_train.ravel())\n",
    "    preds = model.predict(X_test)\n",
    "    \n",
    "    # Compute the score\n",
    "    score = -f1_score(y_test, preds, average='micro')  # Negative because Nevergrad minimizes the objective function\n",
    "\n",
    "    # Print the score and hyperparameters\n",
    "    print(f\"Score: {-score}, Hyperparameters: {hyperparameters}\")\n",
    "\n",
    "    return score\n",
    "\n",
    "# Set up the Nevergrad optimizer\n",
    "parametrization = ng.p.Dict(\n",
    "    n_estimators=ng.p.Scalar(lower=50, upper=500, init=300).set_integer_casting(),\n",
    "    max_depth=ng.p.Scalar(lower=10, upper=100, init=50).set_integer_casting(),\n",
    "    min_samples_split=ng.p.Scalar(lower=2, upper=10, init=2).set_integer_casting(),\n",
    "    min_samples_leaf=ng.p.Scalar(lower=1, upper=10, init=1).set_integer_casting(),\n",
    "    max_features=ng.p.Scalar(lower=10, upper=100, init=50).set_integer_casting(),\n",
    "    learning_rate=ng.p.Scalar(lower=0.01, upper=0.1, init=0.05),\n",
    "    subsample=ng.p.Scalar(lower=0.5, upper=1.0, init=1.0),\n",
    ")\n",
    "optimizer = ng.optimizers.NGOpt(parametrization=parametrization, budget=10)\n",
    "\n",
    "# Run the optimization\n",
    "recommendation = optimizer.minimize(objective_function)\n",
    "# Train the final model with optimized hyperparameters\n",
    "best_hyperparams = recommendation.value\n",
    "print(\"Best hyperparameters:\", best_hyperparams)\n",
    "\n",
    "final_model = RandomForestClassifier(**best_hyperparams)\n",
    "final_model.fit(X_train, y_train.ravel())\n",
    "\n",
    "y_pred = final_model.predict(X_test)\n",
    "f1 = f1_score(y_test, y_pred, average='micro')\n",
    "print(\"F1 Score:\", f1)\n",
    "# Now, final_model is your trained model with optimized hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_out = pd.read_csv('features/X_test_features.csv')\n",
    "out = pd.DataFrame(index=X_out.index, columns=['y'])\n",
    "y_out = predict_model(X_out, final_model)\n",
    "out['y'] = y_out\n",
    "out.to_csv(\"out.csv\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
